"""
Data loading utilities

>>> import testconf
>>> loader( testconf )
"""
import csv, sys
from itertools import *
from atlas import util, hdf
from atlas import commify, info, debug

from mod454.schema import parser
from mod454.schema import Mod454Schema as Schema

@util.stop_on_error
def loader(conf):
    """
    Custom loader for 454 sequencing data
    """
    info( "data label=%s"  % conf.DATA_LABEL )
    info( "data_file='%s'" % conf.DATA_FILE )
    info( "hdf_database='%s'" % conf.HDF_DATABASE )
    def flush( table, label ):
        "Helper function to flush a table"
        if table is not None:
            table.flush() # commit the changes
            size = len(table)
            info("stored label %s with %s rows" % (label, commify(size)))        
    
    # print messages at every CHUNK line
    CHUNK = 10**5 
    last_chrom = table = None
    db = hdf.hdf_open( conf.HDF_DATABASE, mode='a', title='HDF database')
    gp = hdf.create_group( db=db, name=conf.DATA_LABEL, desc='data group', clobber=conf.CLOBBER ) 

    # iterate over the file and insert into table
    for lineno, row in util.dict_reader( conf.DATA_FILE ):
        if (lineno % CHUNK) == 0:
            info("... processed %s lines" % commify(lineno) )    
        chrom = row['chrom']
        if chrom != last_chrom:
            flush( table, last_chrom )
            table = hdf.create_table( db=db, name=chrom, where=gp, schema=Schema, clobber=False )
            last_chrom = chrom
            debug("inserting data for label=%s" % chrom)

        # parser must match Schema
        row = parser( row )
        table.append( [ row ] )

    #flush for last chromosome
    flush(table, chrom)
    info("... finished loading a total of %s lines" % commify(lineno) )
    db.close()

def test(verbose=0):
    """
    Main testrunnner
    """
    import doctest
    doctest.testmod( optionflags=doctest.ELLIPSIS + doctest.NORMALIZE_WHITESPACE, verbose=verbose )

if __name__ == '__main__':
    test(verbose=0)